from scrapy import signals
import scrapy
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
from air_history.settings import logger


class AreaSpiderMiddleware(object):
    def process_request(self, request, spider):
        logger.info('process_request myprint middlewares.py 第五步')
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_experimental_option('excludeSwitches', ['enable-automation']) # 此步骤很重要，设置为开发者模式，防止被各大网站识别出来使用了Selenium
        self.driver = webdriver.Chrome(options=chrome_options)
        # self.driver.maximize_window() # 窗口最大化
        # self.wait = WebDriverWait(self.driver, 10) #超时时长为10s

        # chrome_options = Options()
        # chrome_options.add_argument('--headless')  # 使用无头谷歌浏览器模式
        # chrome_options.add_argument('--disable-gpu')
        # chrome_options.add_argument('--no-sandbox')
        # # 指定谷歌浏览器路径
        # self.driver = webdriver.Chrome(chrome_options=chrome_options,executable_path='D:/home/python3.7.3/Scripts/chromedriver.exe')
        
        if request.url != 'https://www.aqistudy.cn/historydata/': # area_spider.py 的 start_urls
            self.driver.get(request.url)
            time.sleep(1)
            html = self.driver.page_source
            self.driver.quit()
            return scrapy.http.HtmlResponse(url=request.url, body=html.encode('utf-8'), encoding='utf-8',
                                            request=request) # 这里直接返回给spider，不交给downloader了

